{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Evaluation\n",
    "\n",
    "For model performance assessment we want to obtain the distribution of the model quality over 20 independent runs of the training procedure.\n",
    "\n",
    "* [The experiment details](#The-experiment-details)\n",
    "* [Results](#Results)\n",
    "\n",
    "### The experiment detalis\n",
    "Train and test dataset creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "\n",
    "sys.path.append('../../..')\n",
    "\n",
    "from seismicpro.batchflow import Pipeline, B, V, F, C\n",
    "from seismicpro.batchflow.models.tf import UNet\n",
    "from seismicpro.batchflow.research import Research\n",
    "from seismicpro.src import (SeismicDataset, FieldIndex, TraceIndex,\n",
    "                            draw_histogram)\n",
    "\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_raw = '/data/NA/noise_dataset_1/DN02A_LIFT_AMPSCAL.sgy'\n",
    "path_lift = '/data/NA/noise_dataset_1/DN02B_SHOTS_LIFT1.sgy'\n",
    "\n",
    "index = (FieldIndex(name='raw', extra_headers=['offset'], path=path_raw)\n",
    "         .merge(FieldIndex(name='lift', path=path_lift)))\n",
    "\n",
    "train_index = index.create_subset(index.indices[:5])\n",
    "train_set = SeismicDataset(TraceIndex(train_index))\n",
    "test_set = SeismicDataset(TraceIndex(index.create_subset(index.indices[20:21])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define model config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = {\n",
    "    'initial_block/inputs': 'x',\n",
    "    'inputs': dict(x={'shape': (3000, 1)}, \n",
    "                   y={'name':'targets', 'shape': (3000, 1)}),\n",
    "    'body/filters': [16, 32, 64, 128, 256],\n",
    "    'body/encoder': dict(layout='caca', kernel_size=7, activation=tf.nn.elu),\n",
    "    'body/downsample': dict(layout='pd', pool_size=2, pool_strides=2, dropout_rate=0.05),\n",
    "    'body/decoder': dict(layout='caca', kernel_size=7, activation=tf.nn.elu),\n",
    "    'body/upsample': dict(layout='tad', kernel_size=7, strides=2,\n",
    "                          dropout_rate=0.05, activation=tf.nn.elu),\n",
    "    'loss': 'l1',\n",
    "    'optimizer': 'Adam'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define train and test pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_data(batch, **kwagrs):\n",
    "    return {'x': np.expand_dims(np.vstack(batch.raw), -1),\n",
    "            'y': np.expand_dims(np.vstack(batch.lift), -1)}\n",
    "\n",
    "B_SIZE = 16\n",
    "train_pipeline = (Pipeline()\n",
    "                  .load(components=('raw', 'lift'), fmt='segy', tslice=np.arange(3000))\n",
    "                  .init_variable('loss', init_on_each_run=list)\n",
    "                  .init_model('dynamic', UNet, 'unet', model_config)\n",
    "                  .train_model('unet', make_data=make_data,\n",
    "                               fetches='loss', save_to=V('loss', mode='w'))\n",
    "                  .run_later(B_SIZE, n_epochs=None, drop_last=True, shuffle=True)\n",
    "                 ) << train_set\n",
    "\n",
    "test_pipeline = (Pipeline()\n",
    "                    .import_model('unet', C('import_from'))\n",
    "                    .init_variable('res', init_on_each_run=list())\n",
    "                    .init_variable('raw', init_on_each_run=list())\n",
    "                    .init_variable('lift', init_on_each_run=list())\n",
    "                    .load(components=('raw', 'lift'), tslice=np.arange(3000), fmt='segy')\n",
    "                    .update_variable('raw', B('raw'), mode='a')\n",
    "                    .update_variable('lift', B('lift'), mode='a')\n",
    "                    .predict_model('unet', fetches='predictions', make_data=make_data,\n",
    "                                   save_to=V('res', mode='a'))\n",
    "                    .run_later(B_SIZE, n_epochs=1, drop_last=True, shuffle=True)\n",
    "                   ) << test_set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's define the functions for calculating the metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_l1(iteration, experiment, pipeline):\n",
    "    \"\"\" Calculate l1 norm.\"\"\"\n",
    "    _ = iteration\n",
    "    pipeline = experiment[pipeline].pipeline\n",
    "    res = np.squeeze(np.vstack(pipeline.v(\"res\")), axis=-1)\n",
    "    lift = np.vstack(np.concatenate(pipeline.v(\"lift\")))\n",
    "    return np.mean(np.abs(res - lift))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a research object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "research = (Research()\n",
    "            .add_pipeline(train_pipeline, variables='loss', name='train')\n",
    "            .add_pipeline(test_pipeline, name='test_ppl', execute=5,\n",
    "                         run=True, import_from='train')\n",
    "            .add_grid({})\n",
    "            .add_function(get_l1, returns='l1', name='test',\n",
    "                          execute=5, pipeline='test_ppl')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run 20 independent training and test procedures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Research reserach_estimation is starting...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████| 10000/10000 [32:56<00:00,  5.06it/s]"
     ]
    }
   ],
   "source": [
    "research.run(n_reps=20, n_iters=500, name='reserach_estimation', workers=5,\n",
    "             gpu=[1, 2, 3, 6, 7], bar=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "\n",
    "Histogram and a median value of the test metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAF2CAYAAAAr5kJDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHy1JREFUeJzt3X90VtWB7vFnh0jQXrUzDWCjMNaLWJRMVSKrV5ARaCsgI4PTtadWuVOGZW5vtR3ELi0ya9p1e8XqaMTl7S9apP6g6G5HHC8NKBfqMOq0GLRXUnGUUgqiBqhT1HIJE3LuH+8JfcWQ7IT33Ts7+X7WYpGcnB/Pft+cPDnnPTmvybJMAACg/CpiBwAAYKCgdAEACITSBQAgEEoXAIBAKF0AAAKhdAEACITSBbpgjNlhjPm72DlSYoypNMbcZ4z5rTEmM8Zceoz5MmPMNYHjAVFRuhhwjDE/MMb8n2N87egiuEjS3Z7rnZgvf+bxp0zaX0r6rKQ/l/RhSc/6LGSMqTfGrC8q64nlDAnEQOkCXciybG+WZb+PnaMzxpjBsTMcw9mSdmdZ9myWZW9mWXbIc7mTJG2QdFP5ogFxUbpAF44+vWyMmWWMecEYc8AY8ztjzCZjzAX50e2/5LP9Oj9SeypfxhhjvmyM2W6MOWSM+ZUxZv5R2/mQMeZHxpjfG2NajDFfN8bcX3xEbox5yhizLP/aG5J25tM/a4z5uTFmvzFmnzHmJ8aY0UXLnZnn+awx5ok8+8vGmD8zxpxujGnMt/uSMeaSbh6PLseSj/nrks7Kt7nD97HOsmxJlmW3SlrvuwyQGkoX8GSMOU3SjyStlHSepP8iaYmkNkm7JM3KZx2vwmnVK/PPv6BCEX0jX+4fJH3DGDOvaPXLJX1M0kxJUySdIekvOolhJQ2VNFXSJ/NpVZL+p6QL82mHJf2kkyPhr0v6tqTzJW2V9LCk+yV9T9IFkl6S9ENjzAldPAzdjeVKSXdJ2pE/Bhd1sS5gwKmMHQCI5FJjzLs9XObDkk6Q5LIs25FP29rxRWPMW/mHe7Mse7Noua9IujfLsqX5568aY86RtEjSMmPM2Sq8/vmJLMt+mq+rXtInOsnwhqQvZFnW3jEhy7LlxTMYYz4n6bcqFN4zRV+6N8uyx/J5FkvaJOnOLMtW5dNulfS8pHMkNR/jMehyLFmWvZU/roePegwAiCNdDFw/V+GI7+h/XXlR0hOSmo0xq4wxf2uMGdHVAsaYU1Q4at141Jf+WdKZxpiTJJ2bT/tZxxezLPsPSU2drHJzceHm2zg/z/NrY8w7yk87S/qTo5b9v0UfdxTii51MG3YcYwHQBUoXA9X/y7Js29H/ulogy7LDkqarcPr3ORWu0n3FGDOzRJl83vLrPRd15UX3ZL7sXBVObV+Uf3706eX/6GRbnU3j5wJQJuxcQA9kBZuyLFucZdkkFY7y5uZf7rhKd1DR/G9Lek3SpKNW9WeSfp1l2QEVXkuVCq8RSyr8raukcR6RxqjwGu+iLMueyrJsq6Q/kmR6NrLueY4FQBd4TRfwZIy5WIULmJ5U4bXVsyX9qaRl+Sy/kdQuaYYx5hFJrVmW7Zd0m6S7jDGvSnpKhSPl/y7pOknKsuxVY8z/lvRNY8x/k7RX0o2STlH3R7+/kdQq6YvGmLsknanCRU7leqPsLsdyPPIL1U6TVJNPGpW/Pvwmrw+jv+BIF/C3X4Wj0X+S9Kqk+yStUOFqXmVZ1iJpoQoXG72RzycVrhj+e0m3qHBUe7Okr2RZtqxo3XNVuHhpjQpltlvSOkkHuwqUZdk+SdeocNXyLyXdKenLKpR/OfiMpbc+L+kFST/JP1+ef/75Eqwb6BNMlpXrF2IAvWWMGSTpZUmPZ1l2Y+w8AEqD08tAH2CMmaTCVcMvSDpZ0g0qnCr+QbxUAEqN0gX6hkGS/k7SKBWuKG6WNDnLsi1RUwEoKU4vAwAQCBdSAQAQCKULAEAg5XpNl3PWAICBptub0pTtQqrXX3+9XKsum+rqau3bty92jLLrT+M8/fQa7d7d+fdafxpndwbKWAfKOKWBM9b+Ms6ampruZxKnlwEACIbSRdLWrNkbOwIAeKN0AQAIhJtjIGnTpw895mu6ALqXZZkOHjyo9vZ2GVPyN6fqVktLi1pbW4NvtzeyLFNFRYWGDBnS68eK0gWAAezgwYM64YQTVFkZpw4qKys1aNCg7mfsI9ra2nTw4EGdeOKJvVqe08sAMIC1t7dHK9wUVVZWqr2992/iRekiaQsWvBM7ApC0GKeUU3c8jxmli6TdeCOlC6D35s+fr9WrV0uSli9frgkTJuj000/XW2+9VZbtUbpI2oUXDo8dAUA/cdFFF+nhhx/WGWecUbZtULpIWktLOhdgAHi/nTt3atKkSZo/f74mTpyo66+/Xhs3btSsWbM0YcIEvfDCC5KkAwcOaMGCBbr88sv1qU99Sk888YQkadeuXZo9e7Yuu+wyXXbZZXruueckSc8++6w+/elP69prr9WkSZN0/fXXq7t31Rs7dqxGjBhR1vF6vXpurd0h6R1JhyW1OefqyhkKADBw7NixQ9/97nfV0NCgGTNm6LHHHtNjjz2mJ598Uvfee6/uu+8+3XPPPZowYYIaGhq0f/9+XX755brkkktUXV2tlStXasiQIdq+fbuuu+46rVmzRpLU3NysDRs26LTTTtOsWbP03HPPafz48VHH2pNL1iY759K/QSb6ldraQ7EjAP3KXXedrIaGk4983nHXt+nThx6ZtmDBO7rxxnd04YXDj5xtqq09pLVr9+mmm07VihUfODLv5s1v6rTTur7ad8SIERozZowkafTo0Zo4caKMMfroRz+qXbt2SZI2btyodevW6Tvf+Y4kqbW1Vbt379bw4cO1aNEivfTSS6qoqND27duPrPf8888/ck/k8847T7t27UqqdIE+Z+1afg8ESunGG9/p9ALFzm5C8/zzLe+bdscd+3XHHft7tM2qqqojH1dUVGjw4MFHPj58+LCkwo0pli5dqlGjRr1n2bvuuktDhw7VunXr1N7errPOOuvI1zrWI0mDBg1SW1tbj3KVg2/pZpKetNZmkr7rnFt69AzW2npJ9ZLknFN1dXXpUgZSWVnZq9wtsy8uQ5rSGr7q2SMf93ac5dabx/HmLbfo9trFna/veAMdpfgx7Gv66nNaagNlnFK4sba0tPSJv9PtyFBRUaFBgwa956YZlZWVmjx5su6//34tXrxYxhht2bJFtbW1evfdd1VTU6PBgwdr5cqVOnz48JFljTGdrrdYZ9ONMZ3O26GqqqrXz43vIz3RObfbWjtM0jpr7cvOuY3FM+RF3FHGWYpv1dRf3mKqM8Xj6k/jXLnrymOWbqn15cesPz2nXRko45TCjbW1tbVP3BGq4yi0vb1dhw8fVltb25Gj3La2Nn3pS1/SV7/6VV166aVqb2/XiBEj9MADD2jOnDmqr6/XI488osmTJ+ukk046smyWZZ2ut1jx9GXLlulb3/qW9u7dq8mTJ2vKlCm6884735e1tbX1fc+N71v7me6u5jqatfZrkt51zr0/yR9kA+n9dA9fe0UZ0pTWoO89fuTjvvqDqzeP48jGJu2cEea6vuLHsK/pq89pqQ2UcUrhxnrgwAGddNJJZd/OsVRWVvaJ07490dljlpdut3fN6PZPhqy1H7DWntzxsaRPSWruVVIAAAYwn9PLwyWtstZ2zP9D59zasqYCPG2aMi12BADw1m3pOue2S/pYgCxAj23ZP0anDfmX2DEAwAt3pELS5m2+O3YEIGk9va4Hx/eYUboAMIBVVFQkdyFTTG1tbaqo6H11xv/jLABANEOGDNHBgwfV2toa5W3+qqqq1NraGny7vZFlmSoqKjRkyJBer4PSRdJuG3tr7AhA0owxOvHEE6NtfyD9GZjE6WUk7uqRq2JHAABvlC6SNrKxKXYEAPBG6QIAEAilCwBAIJQukjZ12MbuZwKAPoLSRdKW1y2IHQEAvFG6SNrcpobYEQDAG6WLpK3fMyl2BADwRukCABAIpQsAQCCULpK2c0Zd7AgA4I3SRdJW7JwdOwIAeKN0kbSFzYtiRwAAb5QuAACBULoAAARC6SJpy8bdEDsCAHijdJG02lO3xo4AAN4oXSRt/Ia1sSMAgDdKFwCAQChdAAACoXSRtKtGPBo7AgB4o3SRtNtrF8eOAADeKF0kbcbTD8aOAADeKF0krfntMbEjAIA3ShcAgEAoXSRtWNXe2BEAwBuli6Q1TZ0eOwIAeKN0kbSGV+pjRwAAb5QukrZkG6ULIB2ULgAAgVC6AAAEQukiaasnXBM7AgB4o3QBAAiE0kXSZj7zUOwIAOCN0gUAIBBKFwCAQChdJG3+qKWxIwCAN0oXSVswmtIFkA5KF0mrW78mdgQA8EbpIml7WofGjgAA3ihdAAACoXSRtLGnbI0dAQC8UbpIWuPEObEjAIA3ShdJu3nLLbEjAIA3ShdJW7nrytgRAMAbpQsAQCCULgAAgVC6SNqmKdNiRwAAb5QukrZl/5jYEQDAW6XvjNbaQZKaJO12zs0sXyTA37zNd2vnjLrYMQDAS0+OdP9WEnciAACgl7xK11p7hqTLJX2/vHEAAOi/fE8vL5F0k6STjzWDtbZeUr0kOedUXV19/OkCq6ys7FXuljJkKbXicfV2nOXWm8fxtrG3ljzHsfTFx6xDX31OS22gjFMaOGMdKOPs0G3pWmtnStrjnNtsrb30WPM555ZK6nhz02zfvn2lSRhQdXW1Uszto3hc/WmcV49cFWxbffkx60/PaVcGyjilgTPW/jLOmpoar/l8Ti9PkHSFtXaHpIclTbHWPtT7aEDpjGxsih0BALx1e6TrnFsoaaEk5Ue6X3bOXVPmXAAA9Dv8nS4AAIF4/52uJDnnnpL0VFmSAL0wddjG2BEAwBtHukja8roFsSMAgDdKF0mb29QQOwIAeKN0kbT1eybFjgAA3ihdAAACoXQBAAiE0kXSeIchACmhdJG0FTtnx44AAN4oXSRtYfOi2BEAwBulCwBAIJQuAACBULpI2rJxN8SOAADeKF0krfbUrbEjAIA3ShdJG79hbewIAOCN0gUAIBBKFwCAQChdJO2qEY/GjgAA3ihdJO322sWxIwCAN0oXSZvx9IOxIwCAN0oXSWt+e0zsCADgjdIFACAQShdJG1a1N3YEAPBG6SJpTVOnx44AAN4oXSSt4ZX62BEAwBuli6Qt2UbpAkgHpQsAQCCULgAAgVC6SNrqCdfEjgAA3ihdAAACoXSRtJnPPBQ7AgB4o3QBAAiE0gUAIBBKF0mbP2pp7AgA4I3SRdIWjKZ0AaSD0kXS6taviR0BALxRukjantahsSMAgDdKFwCAQChdJG3sKVtjRwAAb5QuktY4cU7sCADgjdJF0m7eckvsCADgjdJF0lbuujJ2BADwRukCABAIpQsAQCCULpK2acq02BEAwBuli6Rt2T8mdgQA8EbpImnzNt8dOwIAeKN0AQAIhNIFACAQShdJu23srbEjAIA3ShdJu3rkqtgRAMAbpYukjWxsih0BALxRugAABELpAgAQCKWLpE0dtjF2BADwVtndDNbaIZI2SqrK5/+xc+6r5Q4G+FhetyB2BADw5nOk2yppinPuY5LOlzTNWvvx8sYC/MxtaogdAQC8dXuk65zLJL2bf3pC/i8rZyjA1/o9k2JHAABvJsu6709r7SBJmyWNkvRN59zNncxTL6lekpxz4w4dOlTiqOVXWVmptra2Hi/XMvviMqSBj5GNTdo5oy52jD5j+KpnY0coq97uoykaKGPtL+McPHiwJJnu5vMq3Q7W2g9KWiXpi8655i5mzV5//XXv9fYV1dXV2rdvX4+XO3ztFWVIAx+U7nsN+t7jsSOUVW/30RQNlLH2l3HW1NRIHqXbo6uXnXO/k/RTSbyJKfoEChdASrotXWvt0PwIV9baEyV9UtLL5Q4G+Fixc3bsCADgzedI98OSfmqtfVHSc5LWOedWlzcW4Gdh86LYEQDAm8/Vyy9KuiBAFgAA+jXuSAUAQCCULpK2bNwNsSMAgDdKF0mrPXVr7AgA4I3SRdLGb1gbOwIAeKN0AQAIhNIFACAQShdJu2rEo7EjAIA3ShdJu712cewIAOCN0kXSZjz9YOwIAOCN0kXSmt8eEzsCAHijdAEACITSRdKGVe2NHQEAvFG6SFrT1OmxIwCAN0oXSWt4pT52BADwRukiaUu2UboA0kHpAgAQCKULAEAglC6StnrCNbEjAIA3ShcAgEAoXSRt5jMPxY4AAN4oXQAAAqF0AQAIhNJF0uaPWho7AgB4o3SRtAWjKV0A6aB0kbS69WtiRwAAb5QukrandWjsCADgjdIFACAQShdJG3vK1tgRAMAbpYukNU6cEzsCAHijdJG0m7fcEjsCAHijdJG0lbuujB0BALxRugAABELpAgAQCKWLpG2aMi12BADwRukiaVv2j4kdAQC8UbpI2rzNd8eOAADeKF0AAAKhdAEACITSRdJuG3tr7AgA4I3SRdKuHrkqdgQA8EbpImkjG5tiRwAAb5QuAACBULoAAARC6SJpU4dtjB0BALxRukja8roFsSMAgDdKF0mb29QQOwIAeKN0kbT1eybFjgAA3ihdAAACoXQBAAiE0kXSds6oix0BALxRukjaip2zY0cAAG+ULpK2sHlR7AgA4I3SBQAgkMruZrDWjpD0gKThkjJJS51z95Q7GAAA/Y3PkW6bpBudc+dK+rik66y155Y3FuBn2bgbYkcAAG/dlq5z7g3n3PP5x+9I2irp9HIHA3zUnro1dgQA8Naj13SttWdKukDSz8uSBuih8RvWxo4AAN66fU23g7X2P0n6R0nznXNvd/L1ekn1kuScU3V1dclChlJZWdmr3C1lyAL0Rl/f71pmX3x8y5coR1eGr3o2wFa619ufR6kZKOPs4FW61toTVCjcFc65Rzubxzm3VNLS/NNs3759pUkYUHV1tVLMDXTg+/f49ZXHcKD8POov46ypqfGar9vTy9ZaI2mZpK3OOd7SBX3KVSM6/R0QAPoknyPdCZLmSNpirf1FPu0W51xj+WIBfm6vXRw7AgB467Z0nXNPSzIBsgA9NuPpB9U4cU7sGADghTtSIWnNb4+JHQEAvFG6AAAEQukiacOq9saOAADeKF0krWnq9NgRAMAbpYukNbxSHzsCAHijdJG0JdsoXQDpoHQBAAiE0gUAIBBKF0lbPeGa2BEAwBulCwBAIJQukjbzmYdiRwAAb5QuAACBULoAAARC6SJp80ctjR0BALxRukjagtGULoB0ULpIWt36NbEjAIA3ShdJ29M6NHYEAPBG6QIAEAili6SNPWVr7AgA4I3SRdIaJ86JHQEAvFG6SNrNW26JHQEAvFG6SNrKXVfGjgAA3ihdAAACoXQBAAiE0kXSNk2ZFjsCAHijdJG0LfvHxI4AAN4oXSRt3ua7Y0cAAG+ULgAAgVC6AAAEQukiabeNvTV2BADwRukiaVePXBU7AgB4o3SRtJGNTbEjAIA3ShcAgEAoXQAAAqF0kbSpwzbGjgAA3ihdJG153YLYEQDAG6WLpM1taogdAQC8UbpI2vo9k2JHAABvlC4AAIFQugAABELpImk7Z9TFjgAA3ihdJG3FztmxIwCAN0oXSVvYvCh2BADwRukCABAIpQsAQCCULpK2bNwNsSMAgDdKF0mrPXVr7AgA4I3SRdLGb1gbOwIAeKN0AQAIhNIFACAQShdJu2rEo7EjAIA3ShdJu712cewIAOCN0kXSZjz9YOwIAOCtsrsZrLX3SZopaY9zbmz5IwH+mt8eEzsCAHjzOdL9gaRpZc4BAEC/123pOuc2SnorQBagx4ZV7Y0dAQC88ZouktY0dXrsCADgrdvXdH1Za+sl1UuSc07V1dWlWrVaZl9csnV1uZ0gW0EpNbxSrwWjl8aO0WeUcr8rhxT2sb7yGFZWVvaZLD3Vk5/Zsb4nhq96Nsp2S1a6zrmlkjp++mX79u0r1aqBY1qyjdItxn53/PrKY1hdXd1nsvRHpX5sa2pqvObj9DIAAIF0W7rW2pWS/lXSOdba16y188ofCwCA/qfb08vOuatCBAF6Y/WEa2JHAABvnF4GACAQShdJm/nMQ7EjAIA3ShcAgEAoXQAAAqF0kbT5o/gbXQDpoHSRNG6MASAllC6SVrd+TewIAOCN0kXS9rQOjR0BALxRugAABELpImljT9kaOwIAeKN0kbTGiXNiRwAAb5QuknbzlltiRwAAb5QukrZy15WxIwCAN0oXAIBAKF0AAAKhdJG0TVOmxY4AAN4oXSRty/4xsSMAgDdKF0mbt/nu2BEAwBulCwBAIJQuAACBULpI2m1jb40dAQC8UbpI2tUjV8WOAADeKF0kbWRjU+wIAOCN0gUAIBBKFwCAQChdJG3qsI2xIwCAN0oXSVtetyB2BADwRukiaXObGmJHAABvlC6Stn7PpNgRAMAbpQsAQCCULgAAgVC6SNrOGXWxIwCAN0oXSVuxc3bsCADgjdJF0hY2L4odAQC8UboAAARC6QIAEAili6QtG3dD7AgA4I3SRdJqT90aOwIAeKN0kbTxG9bGjgAA3ihdAAACoXQBAAiE0kXSrhrxaOwIAOCN0kXSbq9dHDsCAHijdJG0GU8/GDsCAHijdJG05rfHxI4AAN4oXQAAAqF0kbRhVXtjRwAAb5QuktY0dXrsCADgjdJF0hpeqY8dAQC8UbpI2pJtlC6AdFC6AAAEQukCABAIpYukrZ5wTewIAOCt0mcma+00SfdIGiTp+865b5Q1FQAA/VC3R7rW2kGSvilpuqRzJV1lrT233MEAHzOfeSh2BADw5nN6ebykbc657c65Q5IeljSrvLEAAOh/fEr3dEm7ij5/LZ8GAAB6wOs1XR/W2npJ9ZLknFNNTU2pVi39pKl060K/kkmS+P5IBvtyj5T052hIPM/H5HOku1vSiKLPz8invYdzbqlzrs45VyfJpPjPWrs5dgbGyTgZK+McSGPtZ+Psls+R7nOSzrbWfkSFsv2MpM/6rBwAAPxBt0e6zrk2SddLekLS1sIk98tyBwMAoL/xek3XOdcoqbHMWfqCpbEDBMI4+5+BMtaBMk5p4Ix1oIxTkmSyLIudAQCAAYHbQAIAEEjJ/mQolu5uUWmtrZL0gKRxkn4r6a+cczvyry2UNE/SYUlfcs49kU//oKTvSxqrwl+l/I1z7l+ttX8s6RFJZ0raIck65/7dWmvyDDMkHZD0Oefc8wmP8x8k/bmkQ5J+JWmuc+531tozVXhd/9/yzf7MOff5Uo4zwli/JulaSXvz1d+Sv5xyzHUlOs5HJJ2Tr/qDkn7nnDs/xHNa6nFaa89RYT/scJakv3fOLYm5j0YYa7T9NPA4v6ZI+2g5JH2k63mLynmS/t05N0rS3ZJuz5c9V4Ursc+TNE3St/L1SYVvprXOuY9K+pgK38CS9BVJ651zZ0tan3+ufPtn5//qJX078XGukzTWOfenkl6RtLBoO79yzp2f/ytH4YYeqyTdXTSmRo91JTdO59xfdYxR0j9KerRoO2V7TssxTufcvxWNZZwKJboqX1eUfTTSWKPspxHGKUXYR8sl6dKV3y0qZ0m6P//4x5Km5r/1zpL0sHOu1Tn3a0nbJI231p4qaZKkZZLknDvknPtdJ+u6X9JfFE1/wDmXOed+JumD1toPpzpO59yT+VXrkvQzFf42O5TQz+mxdLquEoyvQ5Rx5stbSStLOJaulHycRy07VYWC+U0n6wq5j0qBxxpxPw39nB5LuffRski9dH1uUXlknvwbdL+kD3Wx7EdUOI2x3Fr7grX2+9baD+TzDHfOvZF//Kak4T3IcTxCj7PY30haU/T5R/L5/9lae8lxjOlYYoz1emvti9ba+6y1f9SDHMcj1nN6iaQW59yrRdPK+ZyWY5zFPqP3/gIRax/13UYpx1os5H4aY5wx9tGySL10y6FS0oWSvu2cu0DS7/WHU1RHOOcyddyFME3djtNau0hSm6QV+aQ3JI3M518g6YfW2lPCRe61rsb6bUn/WdL5KozvrigJS8Pne/cqvfcHWqrPqay1gyVdIelHnX29H+yjR3Q11n60nx5rnP1pH02+dH1uUXlkHmttpaRTVXhh/1jLvibpNefcz/PpP1bhB5kktXScksr/39ODHMcj9Dhlrf2cpJmSrs5/eCk/jfPb/OPNKly8Mfr4h9f5OI7K2+k8xztW51yLc+6wc65d0vf0h9NT/fE5rZR0pYouWAnwnJZjnB2mS3reOddSNC3WPuq7jVKONdZ+GnScEffRski9dI/cojL/Dekzkh4/ap7HJf11/vGnJW3Ivzkfl/QZa22VLdzi8mxJm5xzb0ralV9NJxVeX3ipk3X9taR/Kpr+X621xlr7cUn7i05xJTfO/MrEmyRd4Zw70LEBa+3QjgsVrLVn5evaXsJxxhhr8et6syU1F23jfetKdZy5T0h62Tn3WseEAM9pycdZtNzRR+1HryvkPioFHmvE/TT0OGPto2WRdOm6Y9yi0lr7P6y1V+SzLZP0IWvtNhVOtXwlX/aXkpwKP5TWSrrOOXc4X+aLklZYa19U4ZTG4nz6NyR90lr7qgo/wDouk29U4Zt6mwq/iX0h8XH+L0knS1pnrf2FtfY7+fRJkl601v5ChaOozzvn3kp8rHdYa7fk0ydLusFjXSmOU+r8tbKyPqflGmf+WvUn9d6rsKVI+2iksUbZTyOMM8o+Wi7ckQoAgECSPtIFACAllC4AAIFQugAABELpAgAQCKULAEAglC4AAIFQugAABELpAgAQyP8HlWLQ28Pq00EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average value (Median) is 0.006435\n",
      "Std is 0.0004004\n"
     ]
    }
   ],
   "source": [
    "df = research.load_results(use_alias=True)\n",
    "draw_histogram(df, 'test/l1', 100)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
